
<!DOCTYPE html>

<html xmlns="http://www.w3.org/1999/xhtml">
  <head>
    <meta charset="utf-8" />
    <title>UCTB.dataset.data_loader &#8212; UCTB  documentation</title>
    <link rel="stylesheet" href="../../../_static/nature.css" type="text/css" />
    <link rel="stylesheet" href="../../../_static/pygments.css" type="text/css" />
    <script type="text/javascript" id="documentation_options" data-url_root="../../../" src="../../../_static/documentation_options.js"></script>
    <script type="text/javascript" src="../../../_static/jquery.js"></script>
    <script type="text/javascript" src="../../../_static/underscore.js"></script>
    <script type="text/javascript" src="../../../_static/doctools.js"></script>
    <script type="text/javascript" src="../../../_static/language_data.js"></script>
    <link rel="index" title="Index" href="../../../genindex.html" />
    <link rel="search" title="Search" href="../../../search.html" /> 
  </head><body>
    <div class="related" role="navigation" aria-label="related navigation">
      <h3>Navigation</h3>
      <ul>
        <li class="right" style="margin-right: 10px">
          <a href="../../../genindex.html" title="General Index"
             accesskey="I">index</a></li>
        <li class="right" >
          <a href="../../../py-modindex.html" title="Python Module Index"
             >modules</a> |</li>
        <li class="nav-item nav-item-0"><a href="../../../index.html">UCTB  documentation</a> &#187;</li>
          <li class="nav-item nav-item-1"><a href="../../index.html" accesskey="U">Module code</a> &#187;</li> 
      </ul>
    </div>  

    <div class="document">
      <div class="documentwrapper">
        <div class="bodywrapper">
          <div class="body" role="main">
            
  <h1>Source code for UCTB.dataset.data_loader</h1><div class="highlight"><pre>
<span></span><span class="kn">import</span> <span class="nn">os</span>
<span class="kn">import</span> <span class="nn">copy</span>
<span class="kn">import</span> <span class="nn">datetime</span>
<span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>

<span class="kn">from</span> <span class="nn">dateutil.parser</span> <span class="k">import</span> <span class="n">parse</span>
<span class="kn">from</span> <span class="nn">sklearn.metrics.pairwise</span> <span class="k">import</span> <span class="n">cosine_similarity</span>
<span class="kn">from</span> <span class="nn">scipy.stats</span> <span class="k">import</span> <span class="n">pearsonr</span>

<span class="kn">from</span> <span class="nn">..preprocess.time_utils</span> <span class="k">import</span> <span class="n">is_work_day_china</span><span class="p">,</span> <span class="n">is_work_day_america</span><span class="p">,</span> <span class="n">is_valid_date</span>
<span class="kn">from</span> <span class="nn">..preprocess</span> <span class="k">import</span> <span class="n">MoveSample</span><span class="p">,</span> <span class="n">SplitData</span><span class="p">,</span> <span class="n">ST_MoveSample</span><span class="p">,</span> <span class="n">Normalizer</span>
<span class="kn">from</span> <span class="nn">..model_unit</span> <span class="k">import</span> <span class="n">GraphBuilder</span>

<span class="kn">from</span> <span class="nn">.dataset</span> <span class="k">import</span> <span class="n">DataSet</span>


<div class="viewcode-block" id="GridTrafficLoader"><a class="viewcode-back" href="../../../UCTB.dataset.html#UCTB.dataset.data_loader.GridTrafficLoader">[docs]</a><span class="k">class</span> <span class="nc">GridTrafficLoader</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>

    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
                 <span class="n">dataset</span><span class="p">,</span>
                 <span class="n">city</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
                 <span class="n">data_range</span><span class="o">=</span><span class="s1">&#39;all&#39;</span><span class="p">,</span>
                 <span class="n">train_data_length</span><span class="o">=</span><span class="s1">&#39;all&#39;</span><span class="p">,</span>
                 <span class="n">test_ratio</span><span class="o">=</span><span class="mf">0.1</span><span class="p">,</span>
                 <span class="n">closeness_len</span><span class="o">=</span><span class="mi">6</span><span class="p">,</span>
                 <span class="n">period_len</span><span class="o">=</span><span class="mi">7</span><span class="p">,</span>
                 <span class="n">trend_len</span><span class="o">=</span><span class="mi">4</span><span class="p">,</span>
                 <span class="n">target_length</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
                 <span class="n">normalize</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
                 <span class="n">workday_parser</span><span class="o">=</span><span class="n">is_work_day_america</span><span class="p">,</span>
                 <span class="n">data_dir</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">dataset</span> <span class="o">=</span> <span class="n">DataSet</span><span class="p">(</span><span class="n">dataset</span><span class="p">,</span> <span class="n">city</span><span class="p">,</span> <span class="n">data_dir</span><span class="o">=</span><span class="n">data_dir</span><span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">daily_slots</span> <span class="o">=</span> <span class="mi">24</span> <span class="o">*</span> <span class="mi">60</span> <span class="o">/</span> <span class="bp">self</span><span class="o">.</span><span class="n">dataset</span><span class="o">.</span><span class="n">time_fitness</span>

        <span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">data_range</span><span class="p">)</span> <span class="ow">is</span> <span class="nb">str</span> <span class="ow">and</span> <span class="n">data_range</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s1">&#39;all&#39;</span><span class="p">:</span>
            <span class="n">data_range</span> <span class="o">=</span> <span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataset</span><span class="o">.</span><span class="n">grid_traffic</span><span class="p">)]</span>
        <span class="k">elif</span> <span class="nb">type</span><span class="p">(</span><span class="n">data_range</span><span class="p">)</span> <span class="ow">is</span> <span class="nb">float</span><span class="p">:</span>
            <span class="n">data_range</span> <span class="o">=</span> <span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="nb">int</span><span class="p">(</span><span class="n">data_range</span> <span class="o">*</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataset</span><span class="o">.</span><span class="n">grid_traffic</span><span class="p">))]</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="n">data_range</span> <span class="o">=</span> <span class="p">[</span><span class="nb">int</span><span class="p">(</span><span class="n">data_range</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">daily_slots</span><span class="p">),</span> <span class="nb">int</span><span class="p">(</span><span class="n">data_range</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">daily_slots</span><span class="p">)]</span>

        <span class="n">num_time_slots</span> <span class="o">=</span> <span class="n">data_range</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> <span class="o">-</span> <span class="n">data_range</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">traffic_data</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">dataset</span><span class="o">.</span><span class="n">grid_traffic</span><span class="p">[</span><span class="n">data_range</span><span class="p">[</span><span class="mi">0</span><span class="p">]:</span><span class="n">data_range</span><span class="p">[</span><span class="mi">1</span><span class="p">],</span> <span class="p">:]</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">float32</span><span class="p">)</span>

        <span class="c1"># external feature</span>
        <span class="n">external_feature</span> <span class="o">=</span> <span class="p">[]</span>
        <span class="c1"># weather</span>
        <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataset</span><span class="o">.</span><span class="n">external_feature_weather</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
            <span class="n">external_feature</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataset</span><span class="o">.</span><span class="n">external_feature_weather</span><span class="p">[</span><span class="n">data_range</span><span class="p">[</span><span class="mi">0</span><span class="p">]:</span><span class="n">data_range</span><span class="p">[</span><span class="mi">1</span><span class="p">]])</span>
        <span class="c1"># Weekday Feature</span>
        <span class="n">weekday_feature</span> <span class="o">=</span> <span class="p">[[</span><span class="mi">1</span> <span class="k">if</span> <span class="n">workday_parser</span><span class="p">(</span><span class="n">parse</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataset</span><span class="o">.</span><span class="n">time_range</span><span class="p">[</span><span class="mi">1</span><span class="p">])</span>
                                                <span class="o">+</span> <span class="n">datetime</span><span class="o">.</span><span class="n">timedelta</span><span class="p">(</span><span class="n">hours</span><span class="o">=</span><span class="n">e</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">dataset</span><span class="o">.</span><span class="n">time_fitness</span> <span class="o">/</span> <span class="mi">60</span><span class="p">))</span> <span class="k">else</span> <span class="mi">0</span><span class="p">]</span> \
                           <span class="k">for</span> <span class="n">e</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">data_range</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">num_time_slots</span> <span class="o">+</span> <span class="n">data_range</span><span class="p">[</span><span class="mi">0</span><span class="p">])]</span>
        <span class="c1"># Hour Feature</span>
        <span class="n">hour_feature</span> <span class="o">=</span> <span class="p">[[(</span><span class="n">parse</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataset</span><span class="o">.</span><span class="n">time_range</span><span class="p">[</span><span class="mi">1</span><span class="p">])</span> <span class="o">+</span>
                          <span class="n">datetime</span><span class="o">.</span><span class="n">timedelta</span><span class="p">(</span><span class="n">hours</span><span class="o">=</span><span class="n">e</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">dataset</span><span class="o">.</span><span class="n">time_fitness</span> <span class="o">/</span> <span class="mi">60</span><span class="p">))</span><span class="o">.</span><span class="n">hour</span> <span class="o">/</span> <span class="mf">24.0</span><span class="p">]</span>
                        <span class="k">for</span> <span class="n">e</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">data_range</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">num_time_slots</span> <span class="o">+</span> <span class="n">data_range</span><span class="p">[</span><span class="mi">0</span><span class="p">])]</span>

        <span class="n">external_feature</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">weekday_feature</span><span class="p">)</span>
        <span class="n">external_feature</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">hour_feature</span><span class="p">)</span>
        <span class="n">external_feature</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">concatenate</span><span class="p">(</span><span class="n">external_feature</span><span class="p">,</span> <span class="n">axis</span><span class="o">=-</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">float32</span><span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">height</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">width</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">traffic_data</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">],</span> <span class="bp">self</span><span class="o">.</span><span class="n">traffic_data</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">2</span><span class="p">]</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">external_dim</span> <span class="o">=</span> <span class="n">external_feature</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>

        <span class="k">if</span> <span class="n">test_ratio</span> <span class="o">&gt;</span> <span class="mi">1</span> <span class="ow">or</span> <span class="n">test_ratio</span> <span class="o">&lt;</span> <span class="mi">0</span><span class="p">:</span>
            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">&#39;test_ratio &#39;</span><span class="p">)</span>
        <span class="n">train_test_ratio</span> <span class="o">=</span> <span class="p">[</span><span class="mi">1</span> <span class="o">-</span> <span class="n">test_ratio</span><span class="p">,</span> <span class="n">test_ratio</span><span class="p">]</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">train_data</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">test_data</span> <span class="o">=</span> <span class="n">SplitData</span><span class="o">.</span><span class="n">split_data</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">traffic_data</span><span class="p">,</span> <span class="n">train_test_ratio</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">train_ef</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">test_ef</span> <span class="o">=</span> <span class="n">SplitData</span><span class="o">.</span><span class="n">split_data</span><span class="p">(</span><span class="n">external_feature</span><span class="p">,</span> <span class="n">train_test_ratio</span><span class="p">)</span>

        <span class="c1"># Normalize the traffic data</span>
        <span class="k">if</span> <span class="n">normalize</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">normalizer</span> <span class="o">=</span> <span class="n">Normalizer</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">train_data</span><span class="p">)</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">train_data</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">normalizer</span><span class="o">.</span><span class="n">min_max_normal</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">train_data</span><span class="p">)</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">test_data</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">normalizer</span><span class="o">.</span><span class="n">min_max_normal</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">test_data</span><span class="p">)</span>

        <span class="k">if</span> <span class="n">train_data_length</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">!=</span> <span class="s1">&#39;all&#39;</span><span class="p">:</span>
            <span class="n">train_day_length</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">train_data_length</span><span class="p">)</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">train_data</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">train_data</span><span class="p">[</span><span class="o">-</span><span class="nb">int</span><span class="p">(</span><span class="n">train_day_length</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">daily_slots</span><span class="p">):]</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">train_ef</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">train_ef</span><span class="p">[</span><span class="o">-</span><span class="nb">int</span><span class="p">(</span><span class="n">train_day_length</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">daily_slots</span><span class="p">):]</span>

        <span class="c1"># expand the test data</span>
        <span class="n">expand_start_index</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">train_data</span><span class="p">)</span> <span class="o">-</span> <span class="nb">max</span><span class="p">(</span><span class="nb">int</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">daily_slots</span> <span class="o">*</span> <span class="n">period_len</span><span class="p">),</span>
                                                        <span class="nb">int</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">daily_slots</span> <span class="o">*</span> <span class="mi">7</span> <span class="o">*</span> <span class="n">trend_len</span><span class="p">),</span> <span class="n">closeness_len</span><span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">test_data</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">vstack</span><span class="p">([</span><span class="bp">self</span><span class="o">.</span><span class="n">train_data</span><span class="p">[</span><span class="n">expand_start_index</span><span class="p">:],</span> <span class="bp">self</span><span class="o">.</span><span class="n">test_data</span><span class="p">])</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">test_ef</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">vstack</span><span class="p">([</span><span class="bp">self</span><span class="o">.</span><span class="n">train_ef</span><span class="p">[</span><span class="n">expand_start_index</span><span class="p">:],</span> <span class="bp">self</span><span class="o">.</span><span class="n">test_ef</span><span class="p">])</span>

        <span class="k">assert</span> <span class="nb">type</span><span class="p">(</span><span class="n">closeness_len</span><span class="p">)</span> <span class="ow">is</span> <span class="nb">int</span> <span class="ow">and</span> <span class="n">closeness_len</span> <span class="o">&gt;=</span> <span class="mi">0</span>
        <span class="k">assert</span> <span class="nb">type</span><span class="p">(</span><span class="n">period_len</span><span class="p">)</span> <span class="ow">is</span> <span class="nb">int</span> <span class="ow">and</span> <span class="n">period_len</span> <span class="o">&gt;=</span> <span class="mi">0</span>
        <span class="k">assert</span> <span class="nb">type</span><span class="p">(</span><span class="n">trend_len</span><span class="p">)</span> <span class="ow">is</span> <span class="nb">int</span> <span class="ow">and</span> <span class="n">trend_len</span> <span class="o">&gt;=</span> <span class="mi">0</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">closeness_len</span> <span class="o">=</span> <span class="n">closeness_len</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">period_len</span> <span class="o">=</span> <span class="n">period_len</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">trend_len</span> <span class="o">=</span> <span class="n">trend_len</span>

        <span class="c1"># init move sample obj</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">st_move_sample</span> <span class="o">=</span> <span class="n">ST_MoveSample</span><span class="p">(</span><span class="n">closeness_len</span><span class="o">=</span><span class="n">closeness_len</span><span class="p">,</span>
                                            <span class="n">period_len</span><span class="o">=</span><span class="n">period_len</span><span class="p">,</span>
                                            <span class="n">trend_len</span><span class="o">=</span><span class="n">trend_len</span><span class="p">,</span> <span class="n">target_length</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">daily_slots</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">daily_slots</span><span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">train_closeness</span><span class="p">,</span> \
        <span class="bp">self</span><span class="o">.</span><span class="n">train_period</span><span class="p">,</span> \
        <span class="bp">self</span><span class="o">.</span><span class="n">train_trend</span><span class="p">,</span> \
        <span class="bp">self</span><span class="o">.</span><span class="n">train_y</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">st_move_sample</span><span class="o">.</span><span class="n">move_sample</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">train_data</span><span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">test_closeness</span><span class="p">,</span> \
        <span class="bp">self</span><span class="o">.</span><span class="n">test_period</span><span class="p">,</span> \
        <span class="bp">self</span><span class="o">.</span><span class="n">test_trend</span><span class="p">,</span> \
        <span class="bp">self</span><span class="o">.</span><span class="n">test_y</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">st_move_sample</span><span class="o">.</span><span class="n">move_sample</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">test_data</span><span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">train_closeness</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">train_closeness</span><span class="o">.</span><span class="n">squeeze</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">train_period</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">train_period</span><span class="o">.</span><span class="n">squeeze</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">train_trend</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">train_trend</span><span class="o">.</span><span class="n">squeeze</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">test_closeness</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">test_closeness</span><span class="o">.</span><span class="n">squeeze</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">test_period</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">test_period</span><span class="o">.</span><span class="n">squeeze</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">test_trend</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">test_trend</span><span class="o">.</span><span class="n">squeeze</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">train_sequence_len</span> <span class="o">=</span> <span class="nb">max</span><span class="p">((</span><span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">train_closeness</span><span class="p">),</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">train_period</span><span class="p">),</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">train_trend</span><span class="p">)))</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">test_sequence_len</span> <span class="o">=</span> <span class="nb">max</span><span class="p">((</span><span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">test_closeness</span><span class="p">),</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">test_period</span><span class="p">),</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">test_trend</span><span class="p">)))</span>

        <span class="c1"># external feature</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">train_ef</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">train_ef</span><span class="p">[</span><span class="o">-</span><span class="bp">self</span><span class="o">.</span><span class="n">train_sequence_len</span> <span class="o">-</span> <span class="n">target_length</span><span class="p">:</span> <span class="o">-</span><span class="n">target_length</span><span class="p">]</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">test_ef</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">test_ef</span><span class="p">[</span><span class="o">-</span><span class="bp">self</span><span class="o">.</span><span class="n">test_sequence_len</span> <span class="o">-</span> <span class="n">target_length</span><span class="p">:</span> <span class="o">-</span><span class="n">target_length</span><span class="p">]</span></div>


<div class="viewcode-block" id="NodeTrafficLoader"><a class="viewcode-back" href="../../../UCTB.dataset.html#UCTB.dataset.data_loader.NodeTrafficLoader">[docs]</a><span class="k">class</span> <span class="nc">NodeTrafficLoader</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;The data loader that extracts and processes data from a :obj:`DataSet` object.</span>

<span class="sd">    Args:</span>
<span class="sd">        dataset (str): A string containing path of the dataset pickle file or a string of name of the dataset.</span>
<span class="sd">        city (:obj:`str` or ``None``): ``None`` if dataset is file path, or a string of name of the city.</span>
<span class="sd">            Default: ``None``</span>
<span class="sd">        data_range: The range of data extracted from ``self.dataset`` to be further used. If set to ``&#39;all&#39;``, all data in</span>
<span class="sd">            ``self.dataset`` will be used. If set to a float between 0.0 and 1.0, the relative former proportion of data in</span>
<span class="sd">            ``self.dataset`` will be used. If set to a list of two integers ``[start, end]``, the data from *start* day to</span>
<span class="sd">            (*end* - 1) day of data in ``self.dataset`` will be used. Default: ``&#39;all&#39;``</span>
<span class="sd">        train_data_length: The length of train data. If set to ``&#39;all&#39;``, all data in the split train set will be used.</span>
<span class="sd">            If set to int, the latest ``train_data_length`` days of data will be used as train set. Default: ``&#39;all&#39;``</span>
<span class="sd">        test_ratio (float): The ratio of test set as data will be split into train set and test set. Default: 0.1</span>
<span class="sd">        closeness_len (int): The length of closeness data history. The former consecutive ``closeness_len`` time slots</span>
<span class="sd">            of data will be used as closeness history. Default: 6</span>
<span class="sd">        period_len (int): The length of period data history. The data of exact same time slots in former consecutive</span>
<span class="sd">            ``period_len`` days will be used as period history. Default: 7</span>
<span class="sd">        trend_len (int): The length of trend data history. The data of exact same time slots in former consecutive</span>
<span class="sd">            ``trend_len`` weeks (every seven days) will be used as trend history. Default: 4</span>
<span class="sd">        target_length (int): The numbers of steps that need prediction by one piece of history data. Have to be 1 now.</span>
<span class="sd">            Default: 1</span>
<span class="sd">        graph (str): Types of graphs used in neural methods. Graphs should be a subset of { ``&#39;Correlation&#39;``,</span>
<span class="sd">            ``&#39;Distance&#39;``, ``&#39;Interaction&#39;``, ``&#39;Line&#39;``, ``&#39;Neighbor&#39;``, ``&#39;Transfer&#39;`` } and concatenated by ``&#39;-&#39;``,</span>
<span class="sd">            and *dataset* should have data of selected graphs. Default: ``&#39;Correlation&#39;``</span>
<span class="sd">        threshold_distance (float): Used in building of distance graph. If distance of two nodes in meters is larger</span>
<span class="sd">            than ``threshold_distance``, the corresponding position of the distance graph will be 1 and otherwise</span>
<span class="sd">            0.the corresponding Default: 1000</span>
<span class="sd">        threshold_correlation (float): Used in building of correlation graph. If the Pearson correlation coefficient is</span>
<span class="sd">            larger than ``threshold_correlation``, the corresponding position of the correlation graph will be 1</span>
<span class="sd">            and otherwise 0. Default: 0</span>
<span class="sd">        threshold_interaction (float): Used in building of interatction graph. If in the latest 12 months, the number of</span>
<span class="sd">            times of interaction between two nodes is larger than ``threshold_interaction``, the corresponding position</span>
<span class="sd">            of the interaction graph will be 1 and otherwise 0. Default: 500</span>
<span class="sd">        normalize (bool): If ``True``, do min-max normalization on data. Default: ``True``</span>
<span class="sd">        workday_parser: Used to build external features to be used in neural methods. Default: ``is_work_day_america``</span>
<span class="sd">        with_lm (bool): If ``True``, data loader will build graphs according to ``graph``. Default: ``True``</span>
<span class="sd">        with_tpe (bool): If ``True``, data loader will build time position embeddings. Default: ``False``</span>
<span class="sd">        data_dir (:obj:`str` or ``None``): The dataset directory. If set to ``None``, a directory will be created. If</span>
<span class="sd">            ``dataset`` is file path, ``data_dir`` should be ``None`` too. Default: ``None``</span>

<span class="sd">    Attributes:</span>
<span class="sd">        dataset (DataSet): The DataSet object storing basic data.</span>
<span class="sd">        daily_slots (int): The number of time slots in one single day.</span>
<span class="sd">        station_number (int): The number of nodes.</span>
<span class="sd">        external_dim (int): The number of dimensions of external features.</span>
<span class="sd">        train_closeness (np.ndarray): The closeness history of train set data. When ``with_tpe`` is ``False``,</span>
<span class="sd">            its shape is [train_time_slot_num, ``station_number``, ``closeness_len``, 1].</span>
<span class="sd">            On the dimension of ``closeness_len``, data are arranged from earlier time slots to later time slots.</span>
<span class="sd">            If ``closeness_len`` is set to 0, train_closeness will be an empty ndarray.</span>
<span class="sd">            ``train_period``, ``train_trend``, ``test_closeness``, ``test_period``, ``test_trend`` have similar shape</span>
<span class="sd">            and construction.</span>
<span class="sd">        train_y (np.ndarray): The train set data. Its shape is [train_time_slot_num, ``station_number``, 1].</span>
<span class="sd">            ``test_y`` has similar shape and construction.</span>
<span class="sd">        LM (list): If ``with_lm`` is ``True``, the list of Laplacian matrices of graphs listed in ``graph``.</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
                 <span class="n">dataset</span><span class="p">,</span>
                 <span class="n">city</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
                 <span class="n">data_range</span><span class="o">=</span><span class="s1">&#39;all&#39;</span><span class="p">,</span>
                 <span class="n">train_data_length</span><span class="o">=</span><span class="s1">&#39;all&#39;</span><span class="p">,</span>
                 <span class="n">test_ratio</span><span class="o">=</span><span class="mf">0.1</span><span class="p">,</span>
                 <span class="n">closeness_len</span><span class="o">=</span><span class="mi">6</span><span class="p">,</span>
                 <span class="n">period_len</span><span class="o">=</span><span class="mi">7</span><span class="p">,</span>
                 <span class="n">trend_len</span><span class="o">=</span><span class="mi">4</span><span class="p">,</span>
                 <span class="n">target_length</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
                 <span class="n">graph</span><span class="o">=</span><span class="s1">&#39;Correlation&#39;</span><span class="p">,</span>
                 <span class="n">threshold_distance</span><span class="o">=</span><span class="mi">1000</span><span class="p">,</span>
                 <span class="n">threshold_correlation</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span>
                 <span class="n">threshold_interaction</span><span class="o">=</span><span class="mi">500</span><span class="p">,</span>
                 <span class="n">normalize</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
                 <span class="n">workday_parser</span><span class="o">=</span><span class="n">is_work_day_america</span><span class="p">,</span>
                 <span class="n">with_lm</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
                 <span class="n">with_tpe</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
                 <span class="n">data_dir</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">dataset</span> <span class="o">=</span> <span class="n">DataSet</span><span class="p">(</span><span class="n">dataset</span><span class="p">,</span> <span class="n">city</span><span class="p">,</span> <span class="n">data_dir</span><span class="o">=</span><span class="n">data_dir</span><span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">daily_slots</span> <span class="o">=</span> <span class="mi">24</span> <span class="o">*</span> <span class="mi">60</span> <span class="o">/</span> <span class="bp">self</span><span class="o">.</span><span class="n">dataset</span><span class="o">.</span><span class="n">time_fitness</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">closeness_len</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">closeness_len</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">period_len</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">period_len</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">trend_len</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">trend_len</span><span class="p">)</span>

        <span class="k">assert</span> <span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">closeness_len</span><span class="p">)</span> <span class="ow">is</span> <span class="nb">int</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">closeness_len</span> <span class="o">&gt;=</span> <span class="mi">0</span>
        <span class="k">assert</span> <span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">period_len</span><span class="p">)</span> <span class="ow">is</span> <span class="nb">int</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">period_len</span> <span class="o">&gt;=</span> <span class="mi">0</span>
        <span class="k">assert</span> <span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">trend_len</span><span class="p">)</span> <span class="ow">is</span> <span class="nb">int</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">trend_len</span> <span class="o">&gt;=</span> <span class="mi">0</span>

        <span class="k">if</span> <span class="nb">type</span><span class="p">(</span><span class="n">data_range</span><span class="p">)</span> <span class="ow">is</span> <span class="nb">str</span> <span class="ow">and</span> <span class="n">data_range</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s1">&#39;all&#39;</span><span class="p">:</span>
            <span class="n">data_range</span> <span class="o">=</span> <span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataset</span><span class="o">.</span><span class="n">node_traffic</span><span class="p">)]</span>
        <span class="k">elif</span> <span class="nb">type</span><span class="p">(</span><span class="n">data_range</span><span class="p">)</span> <span class="ow">is</span> <span class="nb">float</span><span class="p">:</span>
            <span class="n">data_range</span> <span class="o">=</span> <span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="nb">int</span><span class="p">(</span><span class="n">data_range</span> <span class="o">*</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataset</span><span class="o">.</span><span class="n">node_traffic</span><span class="p">))]</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="n">data_range</span> <span class="o">=</span> <span class="p">[</span><span class="nb">int</span><span class="p">(</span><span class="n">data_range</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">daily_slots</span><span class="p">),</span> <span class="nb">int</span><span class="p">(</span><span class="n">data_range</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">daily_slots</span><span class="p">)]</span>

        <span class="n">num_time_slots</span> <span class="o">=</span> <span class="n">data_range</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> <span class="o">-</span> <span class="n">data_range</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>

        <span class="c1"># traffic feature</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">traffic_data_index</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">where</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataset</span><span class="o">.</span><span class="n">node_traffic</span><span class="p">,</span> <span class="n">axis</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">daily_slots</span> <span class="o">&gt;</span> <span class="mi">1</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">traffic_data</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">dataset</span><span class="o">.</span><span class="n">node_traffic</span><span class="p">[</span><span class="n">data_range</span><span class="p">[</span><span class="mi">0</span><span class="p">]:</span><span class="n">data_range</span><span class="p">[</span><span class="mi">1</span><span class="p">],</span> <span class="bp">self</span><span class="o">.</span><span class="n">traffic_data_index</span><span class="p">]</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span>
            <span class="n">np</span><span class="o">.</span><span class="n">float32</span><span class="p">)</span>

        <span class="c1"># external feature</span>
        <span class="n">external_feature</span> <span class="o">=</span> <span class="p">[]</span>
        <span class="c1"># weather</span>
        <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataset</span><span class="o">.</span><span class="n">external_feature_weather</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
            <span class="n">external_feature</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataset</span><span class="o">.</span><span class="n">external_feature_weather</span><span class="p">[</span><span class="n">data_range</span><span class="p">[</span><span class="mi">0</span><span class="p">]:</span><span class="n">data_range</span><span class="p">[</span><span class="mi">1</span><span class="p">]])</span>
        <span class="c1"># Weekday Feature</span>
        <span class="n">weekday_feature</span> <span class="o">=</span> <span class="p">[[</span><span class="mi">1</span> <span class="k">if</span> <span class="n">workday_parser</span><span class="p">(</span><span class="n">parse</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataset</span><span class="o">.</span><span class="n">time_range</span><span class="p">[</span><span class="mi">1</span><span class="p">])</span>
                                                <span class="o">+</span> <span class="n">datetime</span><span class="o">.</span><span class="n">timedelta</span><span class="p">(</span><span class="n">hours</span><span class="o">=</span><span class="n">e</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">dataset</span><span class="o">.</span><span class="n">time_fitness</span> <span class="o">/</span> <span class="mi">60</span><span class="p">))</span> <span class="k">else</span> <span class="mi">0</span><span class="p">]</span> \
                           <span class="k">for</span> <span class="n">e</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">data_range</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">num_time_slots</span> <span class="o">+</span> <span class="n">data_range</span><span class="p">[</span><span class="mi">0</span><span class="p">])]</span>
        <span class="c1"># Hour Feature</span>
        <span class="n">hour_feature</span> <span class="o">=</span> <span class="p">[[(</span><span class="n">parse</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataset</span><span class="o">.</span><span class="n">time_range</span><span class="p">[</span><span class="mi">1</span><span class="p">])</span> <span class="o">+</span>
                          <span class="n">datetime</span><span class="o">.</span><span class="n">timedelta</span><span class="p">(</span><span class="n">hours</span><span class="o">=</span><span class="n">e</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">dataset</span><span class="o">.</span><span class="n">time_fitness</span> <span class="o">/</span> <span class="mi">60</span><span class="p">))</span><span class="o">.</span><span class="n">hour</span> <span class="o">/</span> <span class="mf">24.0</span><span class="p">]</span>
                        <span class="k">for</span> <span class="n">e</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">data_range</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">num_time_slots</span> <span class="o">+</span> <span class="n">data_range</span><span class="p">[</span><span class="mi">0</span><span class="p">])]</span>

        <span class="n">external_feature</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">weekday_feature</span><span class="p">)</span>
        <span class="n">external_feature</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">hour_feature</span><span class="p">)</span>
        <span class="n">external_feature</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">concatenate</span><span class="p">(</span><span class="n">external_feature</span><span class="p">,</span> <span class="n">axis</span><span class="o">=-</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">float32</span><span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">station_number</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">traffic_data</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">external_dim</span> <span class="o">=</span> <span class="n">external_feature</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>

        <span class="k">if</span> <span class="n">test_ratio</span> <span class="o">&gt;</span> <span class="mi">1</span> <span class="ow">or</span> <span class="n">test_ratio</span> <span class="o">&lt;</span> <span class="mi">0</span><span class="p">:</span>
            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">&#39;test_ratio &#39;</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">train_test_ratio</span> <span class="o">=</span> <span class="p">[</span><span class="mi">1</span> <span class="o">-</span> <span class="n">test_ratio</span><span class="p">,</span> <span class="n">test_ratio</span><span class="p">]</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">train_data</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">test_data</span> <span class="o">=</span> <span class="n">SplitData</span><span class="o">.</span><span class="n">split_data</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">traffic_data</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">train_test_ratio</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">train_ef</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">test_ef</span> <span class="o">=</span> <span class="n">SplitData</span><span class="o">.</span><span class="n">split_data</span><span class="p">(</span><span class="n">external_feature</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">train_test_ratio</span><span class="p">)</span>

        <span class="c1"># Normalize the traffic data</span>
        <span class="k">if</span> <span class="n">normalize</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">normalizer</span> <span class="o">=</span> <span class="n">Normalizer</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">train_data</span><span class="p">)</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">train_data</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">normalizer</span><span class="o">.</span><span class="n">min_max_normal</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">train_data</span><span class="p">)</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">test_data</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">normalizer</span><span class="o">.</span><span class="n">min_max_normal</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">test_data</span><span class="p">)</span>

        <span class="k">if</span> <span class="n">train_data_length</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">!=</span> <span class="s1">&#39;all&#39;</span><span class="p">:</span>
            <span class="n">train_day_length</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">train_data_length</span><span class="p">)</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">train_data</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">train_data</span><span class="p">[</span><span class="o">-</span><span class="nb">int</span><span class="p">(</span><span class="n">train_day_length</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">daily_slots</span><span class="p">):]</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">train_ef</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">train_ef</span><span class="p">[</span><span class="o">-</span><span class="nb">int</span><span class="p">(</span><span class="n">train_day_length</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">daily_slots</span><span class="p">):]</span>

        <span class="c1"># expand the test data</span>
        <span class="n">expand_start_index</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">train_data</span><span class="p">)</span> <span class="o">-</span> \
                             <span class="nb">max</span><span class="p">(</span><span class="nb">int</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">daily_slots</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">period_len</span><span class="p">),</span>
                                 <span class="nb">int</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">daily_slots</span> <span class="o">*</span> <span class="mi">7</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">trend_len</span><span class="p">),</span>
                                 <span class="bp">self</span><span class="o">.</span><span class="n">closeness_len</span><span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">test_data</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">vstack</span><span class="p">([</span><span class="bp">self</span><span class="o">.</span><span class="n">train_data</span><span class="p">[</span><span class="n">expand_start_index</span><span class="p">:],</span> <span class="bp">self</span><span class="o">.</span><span class="n">test_data</span><span class="p">])</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">test_ef</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">vstack</span><span class="p">([</span><span class="bp">self</span><span class="o">.</span><span class="n">train_ef</span><span class="p">[</span><span class="n">expand_start_index</span><span class="p">:],</span> <span class="bp">self</span><span class="o">.</span><span class="n">test_ef</span><span class="p">])</span>

        <span class="c1"># init move sample obj</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">st_move_sample</span> <span class="o">=</span> <span class="n">ST_MoveSample</span><span class="p">(</span><span class="n">closeness_len</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">closeness_len</span><span class="p">,</span>
                                            <span class="n">period_len</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">period_len</span><span class="p">,</span>
                                            <span class="n">trend_len</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">trend_len</span><span class="p">,</span> <span class="n">target_length</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">daily_slots</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">daily_slots</span><span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">train_closeness</span><span class="p">,</span> \
        <span class="bp">self</span><span class="o">.</span><span class="n">train_period</span><span class="p">,</span> \
        <span class="bp">self</span><span class="o">.</span><span class="n">train_trend</span><span class="p">,</span> \
        <span class="bp">self</span><span class="o">.</span><span class="n">train_y</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">st_move_sample</span><span class="o">.</span><span class="n">move_sample</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">train_data</span><span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">test_closeness</span><span class="p">,</span> \
        <span class="bp">self</span><span class="o">.</span><span class="n">test_period</span><span class="p">,</span> \
        <span class="bp">self</span><span class="o">.</span><span class="n">test_trend</span><span class="p">,</span> \
        <span class="bp">self</span><span class="o">.</span><span class="n">test_y</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">st_move_sample</span><span class="o">.</span><span class="n">move_sample</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">test_data</span><span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">train_sequence_len</span> <span class="o">=</span> <span class="nb">max</span><span class="p">((</span><span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">train_closeness</span><span class="p">),</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">train_period</span><span class="p">),</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">train_trend</span><span class="p">)))</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">test_sequence_len</span> <span class="o">=</span> <span class="nb">max</span><span class="p">((</span><span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">test_closeness</span><span class="p">),</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">test_period</span><span class="p">),</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">test_trend</span><span class="p">)))</span>

        <span class="c1"># external feature</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">train_ef</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">train_ef</span><span class="p">[</span><span class="o">-</span><span class="bp">self</span><span class="o">.</span><span class="n">train_sequence_len</span> <span class="o">-</span> <span class="n">target_length</span><span class="p">:</span> <span class="o">-</span><span class="n">target_length</span><span class="p">]</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">test_ef</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">test_ef</span><span class="p">[</span><span class="o">-</span><span class="bp">self</span><span class="o">.</span><span class="n">test_sequence_len</span> <span class="o">-</span> <span class="n">target_length</span><span class="p">:</span> <span class="o">-</span><span class="n">target_length</span><span class="p">]</span>

        <span class="k">if</span> <span class="n">with_tpe</span><span class="p">:</span>

            <span class="c1"># Time position embedding</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">closeness_tpe</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="nb">range</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">closeness_len</span> <span class="o">+</span> <span class="mi">1</span><span class="p">),</span> <span class="n">dtype</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">float32</span><span class="p">)</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">period_tpe</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="nb">range</span><span class="p">(</span><span class="mi">1</span> <span class="o">*</span> <span class="nb">int</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">daily_slots</span><span class="p">),</span>
                                             <span class="bp">self</span><span class="o">.</span><span class="n">period_len</span> <span class="o">*</span> <span class="nb">int</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">daily_slots</span><span class="p">)</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span>
                                             <span class="nb">int</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">daily_slots</span><span class="p">)),</span> <span class="n">dtype</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">float32</span><span class="p">)</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">trend_tpe</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="nb">range</span><span class="p">(</span><span class="mi">1</span> <span class="o">*</span> <span class="nb">int</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">daily_slots</span><span class="p">)</span> <span class="o">*</span> <span class="mi">7</span><span class="p">,</span>
                                            <span class="bp">self</span><span class="o">.</span><span class="n">trend_len</span> <span class="o">*</span> <span class="nb">int</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">daily_slots</span><span class="p">)</span> <span class="o">*</span> <span class="mi">7</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span>
                                            <span class="nb">int</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">daily_slots</span><span class="p">)</span> <span class="o">*</span> <span class="mi">7</span><span class="p">),</span> <span class="n">dtype</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">float32</span><span class="p">)</span>

            <span class="bp">self</span><span class="o">.</span><span class="n">train_closeness_tpe</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">tile</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">reshape</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">closeness_tpe</span><span class="p">,</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">]),</span>
                                               <span class="p">[</span><span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">train_closeness</span><span class="p">),</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">traffic_data_index</span><span class="p">),</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">])</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">train_period_tpe</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">tile</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">reshape</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">period_tpe</span><span class="p">,</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">]),</span>
                                            <span class="p">[</span><span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">train_period</span><span class="p">),</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">traffic_data_index</span><span class="p">),</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">])</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">train_trend_tpe</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">tile</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">reshape</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">trend_tpe</span><span class="p">,</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">]),</span>
                                           <span class="p">[</span><span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">train_trend</span><span class="p">),</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">traffic_data_index</span><span class="p">),</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">])</span>

            <span class="bp">self</span><span class="o">.</span><span class="n">test_closeness_tpe</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">tile</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">reshape</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">closeness_tpe</span><span class="p">,</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">]),</span>
                                              <span class="p">[</span><span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">test_closeness</span><span class="p">),</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">traffic_data_index</span><span class="p">),</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">])</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">test_period_tpe</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">tile</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">reshape</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">period_tpe</span><span class="p">,</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">]),</span>
                                           <span class="p">[</span><span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">test_period</span><span class="p">),</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">traffic_data_index</span><span class="p">),</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">])</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">test_trend_tpe</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">tile</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">reshape</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">trend_tpe</span><span class="p">,</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">]),</span>
                                          <span class="p">[</span><span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">test_trend</span><span class="p">),</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">traffic_data_index</span><span class="p">),</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">])</span>

            <span class="bp">self</span><span class="o">.</span><span class="n">tpe_dim</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">train_closeness_tpe</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>

            <span class="c1"># concat temporal feature with time position embedding</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">train_closeness</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">concatenate</span><span class="p">((</span><span class="bp">self</span><span class="o">.</span><span class="n">train_closeness</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">train_closeness_tpe</span><span class="p">,),</span> <span class="n">axis</span><span class="o">=-</span><span class="mi">1</span><span class="p">)</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">train_period</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">concatenate</span><span class="p">((</span><span class="bp">self</span><span class="o">.</span><span class="n">train_period</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">train_period_tpe</span><span class="p">,),</span> <span class="n">axis</span><span class="o">=-</span><span class="mi">1</span><span class="p">)</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">train_trend</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">concatenate</span><span class="p">((</span><span class="bp">self</span><span class="o">.</span><span class="n">train_trend</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">train_trend_tpe</span><span class="p">,),</span> <span class="n">axis</span><span class="o">=-</span><span class="mi">1</span><span class="p">)</span>

            <span class="bp">self</span><span class="o">.</span><span class="n">test_closeness</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">concatenate</span><span class="p">((</span><span class="bp">self</span><span class="o">.</span><span class="n">test_closeness</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">test_closeness_tpe</span><span class="p">,),</span> <span class="n">axis</span><span class="o">=-</span><span class="mi">1</span><span class="p">)</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">test_period</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">concatenate</span><span class="p">((</span><span class="bp">self</span><span class="o">.</span><span class="n">test_period</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">test_period_tpe</span><span class="p">,),</span> <span class="n">axis</span><span class="o">=-</span><span class="mi">1</span><span class="p">)</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">test_trend</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">concatenate</span><span class="p">((</span><span class="bp">self</span><span class="o">.</span><span class="n">test_trend</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">test_trend_tpe</span><span class="p">,),</span> <span class="n">axis</span><span class="o">=-</span><span class="mi">1</span><span class="p">)</span>

        <span class="k">else</span><span class="p">:</span>

            <span class="bp">self</span><span class="o">.</span><span class="n">tpe_dim</span> <span class="o">=</span> <span class="kc">None</span>

        <span class="k">if</span> <span class="n">with_lm</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">AM</span> <span class="o">=</span> <span class="p">[]</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">LM</span> <span class="o">=</span> <span class="p">[]</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">threshold_distance</span> <span class="o">=</span> <span class="n">threshold_distance</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">threshold_correlation</span> <span class="o">=</span> <span class="n">threshold_correlation</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">threshold_interaction</span> <span class="o">=</span> <span class="n">threshold_interaction</span>

            <span class="k">for</span> <span class="n">graph_name</span> <span class="ow">in</span> <span class="n">graph</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;-&#39;</span><span class="p">):</span>
                <span class="n">AM</span><span class="p">,</span> <span class="n">LM</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">build_graph</span><span class="p">(</span><span class="n">graph_name</span><span class="p">)</span>
                <span class="k">if</span> <span class="n">AM</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
                    <span class="bp">self</span><span class="o">.</span><span class="n">AM</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">AM</span><span class="p">)</span>
                <span class="k">if</span> <span class="n">LM</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
                    <span class="bp">self</span><span class="o">.</span><span class="n">LM</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">LM</span><span class="p">)</span>

            <span class="bp">self</span><span class="o">.</span><span class="n">LM</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">LM</span><span class="p">,</span> <span class="n">dtype</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">float32</span><span class="p">)</span>

<div class="viewcode-block" id="NodeTrafficLoader.build_graph"><a class="viewcode-back" href="../../../UCTB.dataset.html#UCTB.dataset.data_loader.NodeTrafficLoader.build_graph">[docs]</a>    <span class="k">def</span> <span class="nf">build_graph</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">graph_name</span><span class="p">):</span>
        <span class="n">AM</span><span class="p">,</span> <span class="n">LM</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="kc">None</span>
        <span class="k">if</span> <span class="n">graph_name</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s1">&#39;distance&#39;</span><span class="p">:</span>
            <span class="n">lat_lng_list</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([[</span><span class="nb">float</span><span class="p">(</span><span class="n">e1</span><span class="p">)</span> <span class="k">for</span> <span class="n">e1</span> <span class="ow">in</span> <span class="n">e</span><span class="p">[</span><span class="mi">2</span><span class="p">:</span><span class="mi">4</span><span class="p">]]</span> <span class="k">for</span> <span class="n">e</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">dataset</span><span class="o">.</span><span class="n">node_station_info</span><span class="p">])</span>
            <span class="n">AM</span> <span class="o">=</span> <span class="n">GraphBuilder</span><span class="o">.</span><span class="n">distance_adjacent</span><span class="p">(</span><span class="n">lat_lng_list</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">traffic_data_index</span><span class="p">],</span>
                                                <span class="n">threshold</span><span class="o">=</span><span class="nb">float</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">threshold_distance</span><span class="p">))</span>
            <span class="n">LM</span> <span class="o">=</span> <span class="n">GraphBuilder</span><span class="o">.</span><span class="n">adjacent_to_laplacian</span><span class="p">(</span><span class="n">AM</span><span class="p">)</span>

        <span class="k">if</span> <span class="n">graph_name</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s1">&#39;interaction&#39;</span><span class="p">:</span>
            <span class="n">monthly_interaction</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">dataset</span><span class="o">.</span><span class="n">node_monthly_interaction</span><span class="p">[:,</span> <span class="bp">self</span><span class="o">.</span><span class="n">traffic_data_index</span><span class="p">,</span> <span class="p">:][:,</span> <span class="p">:,</span>
                                  <span class="bp">self</span><span class="o">.</span><span class="n">traffic_data_index</span><span class="p">]</span>

            <span class="n">monthly_interaction</span><span class="p">,</span> <span class="n">_</span> <span class="o">=</span> <span class="n">SplitData</span><span class="o">.</span><span class="n">split_data</span><span class="p">(</span><span class="n">monthly_interaction</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">train_test_ratio</span><span class="p">)</span>

            <span class="n">annually_interaction</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">sum</span><span class="p">(</span><span class="n">monthly_interaction</span><span class="p">[</span><span class="o">-</span><span class="mi">12</span><span class="p">:],</span> <span class="n">axis</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
            <span class="n">annually_interaction</span> <span class="o">=</span> <span class="n">annually_interaction</span> <span class="o">+</span> <span class="n">annually_interaction</span><span class="o">.</span><span class="n">transpose</span><span class="p">()</span>

            <span class="n">AM</span> <span class="o">=</span> <span class="n">GraphBuilder</span><span class="o">.</span><span class="n">interaction_adjacent</span><span class="p">(</span><span class="n">annually_interaction</span><span class="p">,</span>
                                                   <span class="n">threshold</span><span class="o">=</span><span class="nb">float</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">threshold_interaction</span><span class="p">))</span>
            <span class="n">LM</span> <span class="o">=</span> <span class="n">GraphBuilder</span><span class="o">.</span><span class="n">adjacent_to_laplacian</span><span class="p">(</span><span class="n">AM</span><span class="p">)</span>

        <span class="k">if</span> <span class="n">graph_name</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s1">&#39;correlation&#39;</span><span class="p">:</span>
            <span class="n">AM</span> <span class="o">=</span> <span class="n">GraphBuilder</span><span class="o">.</span><span class="n">correlation_adjacent</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">train_data</span><span class="p">[</span><span class="o">-</span><span class="mi">30</span> <span class="o">*</span> <span class="nb">int</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">daily_slots</span><span class="p">):],</span>
                                                   <span class="n">threshold</span><span class="o">=</span><span class="nb">float</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">threshold_correlation</span><span class="p">))</span>
            <span class="n">LM</span> <span class="o">=</span> <span class="n">GraphBuilder</span><span class="o">.</span><span class="n">adjacent_to_laplacian</span><span class="p">(</span><span class="n">AM</span><span class="p">)</span>

        <span class="k">if</span> <span class="n">graph_name</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s1">&#39;neighbor&#39;</span><span class="p">:</span>
            <span class="n">LM</span> <span class="o">=</span> <span class="n">GraphBuilder</span><span class="o">.</span><span class="n">adjacent_to_laplacian</span><span class="p">(</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">dataset</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;contribute_data&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;graph_neighbors&#39;</span><span class="p">))</span>

        <span class="k">if</span> <span class="n">graph_name</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s1">&#39;line&#39;</span><span class="p">:</span>
            <span class="n">LM</span> <span class="o">=</span> <span class="n">GraphBuilder</span><span class="o">.</span><span class="n">adjacent_to_laplacian</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataset</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;contribute_data&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;graph_lines&#39;</span><span class="p">))</span>

        <span class="k">if</span> <span class="n">graph_name</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span> <span class="o">==</span> <span class="s1">&#39;transfer&#39;</span><span class="p">:</span>
            <span class="n">LM</span> <span class="o">=</span> <span class="n">GraphBuilder</span><span class="o">.</span><span class="n">adjacent_to_laplacian</span><span class="p">(</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">dataset</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;contribute_data&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;graph_transfer&#39;</span><span class="p">))</span>
        <span class="k">return</span> <span class="n">AM</span><span class="p">,</span> <span class="n">LM</span></div></div>


<div class="viewcode-block" id="st_map"><a class="viewcode-back" href="../../../UCTB.dataset.html#UCTB.dataset.data_loader.st_map">[docs]</a><span class="k">def</span> <span class="nf">st_map</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">zoom</span><span class="o">=</span><span class="mi">11</span><span class="p">,</span> <span class="n">style</span><span class="o">=</span><span class="s1">&#39;mapbox://styles/rmetfc/ck1manozn0edb1dpmvtzle2cp&#39;</span><span class="p">,</span> <span class="n">build_order</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">dataset</span><span class="o">.</span><span class="n">node_station_info</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">or</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataset</span><span class="o">.</span><span class="n">node_station_info</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">&#39;No station information found in dataset&#39;</span><span class="p">)</span>

    <span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
    <span class="kn">import</span> <span class="nn">plotly</span>
    <span class="kn">from</span> <span class="nn">plotly.graph_objs</span> <span class="k">import</span> <span class="n">Scattermapbox</span><span class="p">,</span> <span class="n">Layout</span>

    <span class="n">mapboxAccessToken</span> <span class="o">=</span> <span class="s2">&quot;pk.eyJ1Ijoicm1ldGZjIiwiYSI6ImNrMW02YmwxbjAxN24zam9kNGVtMm5raWIifQ.FXKqZCxsFK-dGLLNdeRJHw&quot;</span>

    <span class="c1"># os.environ[&#39;MAPBOX_API_KEY&#39;] = mapboxAccessToken</span>

    <span class="n">lat_lng_name_list</span> <span class="o">=</span> <span class="p">[</span><span class="n">e</span><span class="p">[</span><span class="mi">2</span><span class="p">:]</span> <span class="k">for</span> <span class="n">e</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">dataset</span><span class="o">.</span><span class="n">node_station_info</span><span class="p">]</span>
    <span class="n">build_order</span> <span class="o">=</span> <span class="n">build_order</span> <span class="ow">or</span> <span class="nb">list</span><span class="p">(</span><span class="nb">range</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">dataset</span><span class="o">.</span><span class="n">node_station_info</span><span class="p">)))</span>

    <span class="n">color</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;rgb(255, 0, 0)&#39;</span> <span class="k">for</span> <span class="n">_</span> <span class="ow">in</span> <span class="n">build_order</span><span class="p">]</span>

    <span class="n">lat</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="nb">float</span><span class="p">(</span><span class="n">e</span><span class="p">[</span><span class="mi">2</span><span class="p">])</span> <span class="k">for</span> <span class="n">e</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">dataset</span><span class="o">.</span><span class="n">node_station_info</span><span class="p">])[</span><span class="bp">self</span><span class="o">.</span><span class="n">traffic_data_index</span><span class="p">]</span>
    <span class="n">lng</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="nb">float</span><span class="p">(</span><span class="n">e</span><span class="p">[</span><span class="mi">3</span><span class="p">])</span> <span class="k">for</span> <span class="n">e</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">dataset</span><span class="o">.</span><span class="n">node_station_info</span><span class="p">])[</span><span class="bp">self</span><span class="o">.</span><span class="n">traffic_data_index</span><span class="p">]</span>
    <span class="n">text</span> <span class="o">=</span> <span class="p">[</span><span class="nb">str</span><span class="p">(</span><span class="n">e</span><span class="p">)</span> <span class="k">for</span> <span class="n">e</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">build_order</span><span class="p">))]</span>

    <span class="n">file_name</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">dataset</span><span class="o">.</span><span class="n">dataset</span> <span class="o">+</span> <span class="s1">&#39;-&#39;</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">dataset</span><span class="o">.</span><span class="n">city</span> <span class="o">+</span> <span class="s1">&#39;.html&#39;</span>

    <span class="n">bikeStations</span> <span class="o">=</span> <span class="p">[</span><span class="n">Scattermapbox</span><span class="p">(</span>
        <span class="n">lon</span><span class="o">=</span><span class="n">lng</span><span class="p">,</span>
        <span class="n">lat</span><span class="o">=</span><span class="n">lat</span><span class="p">,</span>
        <span class="n">text</span><span class="o">=</span><span class="n">text</span><span class="p">,</span>
        <span class="n">mode</span><span class="o">=</span><span class="s1">&#39;markers&#39;</span><span class="p">,</span>
        <span class="n">marker</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span>
            <span class="n">size</span><span class="o">=</span><span class="mi">6</span><span class="p">,</span>
            <span class="c1"># color=[&#39;rgb(%s, %s, %s)&#39; % (255,</span>
            <span class="c1">#                 #                             195 - e * 195 / max(build_order),</span>
            <span class="c1">#                 #                             195 - e * 195 / max(build_order)) for e in build_order],</span>
            <span class="n">color</span><span class="o">=</span><span class="n">color</span><span class="p">,</span>
            <span class="n">opacity</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
        <span class="p">))]</span>

    <span class="n">layout</span> <span class="o">=</span> <span class="n">Layout</span><span class="p">(</span>
        <span class="n">title</span><span class="o">=</span><span class="s1">&#39;Bike Station Location &amp; The latest built stations with deeper color&#39;</span><span class="p">,</span>
        <span class="n">autosize</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
        <span class="n">hovermode</span><span class="o">=</span><span class="s1">&#39;closest&#39;</span><span class="p">,</span>
        <span class="n">showlegend</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
        <span class="n">mapbox</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span>
            <span class="n">accesstoken</span><span class="o">=</span><span class="n">mapboxAccessToken</span><span class="p">,</span>
            <span class="n">bearing</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span>
            <span class="n">center</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span>
                <span class="n">lat</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">median</span><span class="p">(</span><span class="n">lat</span><span class="p">),</span>
                <span class="n">lon</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">median</span><span class="p">(</span><span class="n">lng</span><span class="p">)</span>
            <span class="p">),</span>
            <span class="n">pitch</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span>
            <span class="n">zoom</span><span class="o">=</span><span class="n">zoom</span><span class="p">,</span>
            <span class="n">style</span><span class="o">=</span><span class="n">style</span>
        <span class="p">),</span>
    <span class="p">)</span>

    <span class="n">fig</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span><span class="n">data</span><span class="o">=</span><span class="n">bikeStations</span><span class="p">,</span> <span class="n">layout</span><span class="o">=</span><span class="n">layout</span><span class="p">)</span>
    <span class="n">plotly</span><span class="o">.</span><span class="n">offline</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">fig</span><span class="p">,</span> <span class="n">filename</span><span class="o">=</span><span class="n">file_name</span><span class="p">)</span></div>


<div class="viewcode-block" id="make_concat"><a class="viewcode-back" href="../../../UCTB.dataset.html#UCTB.dataset.data_loader.make_concat">[docs]</a><span class="k">def</span> <span class="nf">make_concat</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">node</span><span class="o">=</span><span class="s1">&#39;all&#39;</span><span class="p">,</span> <span class="n">is_train</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;A function to concatenate all closeness, period and trend history data to use as inputs of models.</span>

<span class="sd">    Args:</span>
<span class="sd">        node (int or ``&#39;all&#39;``): To specify the index of certain node. If set to ``&#39;all&#39;``, return the concatenation</span>
<span class="sd">            result of all nodes. If set to an integer, it will be the index of the selected node. Default: ``&#39;all&#39;``</span>
<span class="sd">        is_train (bool): If set to ``True``, ``train_closeness``, ``train_period``, and ``train_trend`` will be</span>
<span class="sd">            concatenated. If set to ``False``, ``test_closeness``, ``test_period``, and ``test_trend`` will be</span>
<span class="sd">            concatenated. Default: True</span>

<span class="sd">    Returns:</span>
<span class="sd">        np.ndarray: Function returns an ndarray with shape as</span>
<span class="sd">        [time_slot_num, ``station_number``, ``closeness_len`` + ``period_len`` + ``trend_len``, 1],</span>
<span class="sd">        and time_slot_num is the temporal length of train set data if ``is_train`` is ``True``</span>
<span class="sd">        or the temporal length of test set data if ``is_train`` is ``False``.</span>
<span class="sd">        On the second dimension, data are arranged as</span>
<span class="sd">        ``earlier closeness -&gt; later closeness -&gt; earlier period -&gt; later period -&gt; earlier trend -&gt; later trend``.</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="k">if</span> <span class="n">is_train</span><span class="p">:</span>
        <span class="n">length</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">train_y</span><span class="p">)</span>
        <span class="n">closeness</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">train_closeness</span>
        <span class="n">period</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">train_period</span>
        <span class="n">trend</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">train_trend</span>
    <span class="k">else</span><span class="p">:</span>
        <span class="n">length</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">test_y</span><span class="p">)</span>
        <span class="n">closeness</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">test_closeness</span>
        <span class="n">period</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">test_period</span>
        <span class="n">trend</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">test_trend</span>
    <span class="k">if</span> <span class="n">node</span> <span class="o">==</span> <span class="s1">&#39;all&#39;</span><span class="p">:</span>
        <span class="n">node</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">range</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">station_number</span><span class="p">))</span>
    <span class="k">else</span><span class="p">:</span>
        <span class="n">node</span> <span class="o">=</span> <span class="p">[</span><span class="n">node</span><span class="p">]</span>
    <span class="n">history</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">([</span><span class="n">length</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">node</span><span class="p">),</span> <span class="bp">self</span><span class="o">.</span><span class="n">closeness_len</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">period_len</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">trend_len</span><span class="p">])</span>
    <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">node</span><span class="p">)):</span>
        <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">closeness_len</span><span class="p">):</span>
            <span class="n">history</span><span class="p">[:,</span> <span class="n">i</span><span class="p">,</span> <span class="n">c</span><span class="p">]</span> <span class="o">=</span> <span class="n">closeness</span><span class="p">[:,</span> <span class="n">node</span><span class="p">[</span><span class="n">i</span><span class="p">],</span> <span class="n">c</span><span class="p">,</span> <span class="o">-</span><span class="mi">1</span><span class="p">]</span>
        <span class="k">for</span> <span class="n">p</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">period_len</span><span class="p">):</span>
            <span class="n">history</span><span class="p">[:,</span> <span class="n">i</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">closeness_len</span> <span class="o">+</span> <span class="n">p</span><span class="p">]</span> <span class="o">=</span> <span class="n">period</span><span class="p">[:,</span> <span class="n">node</span><span class="p">[</span><span class="n">i</span><span class="p">],</span> <span class="n">p</span><span class="p">,</span> <span class="o">-</span><span class="mi">1</span><span class="p">]</span>
        <span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">trend_len</span><span class="p">):</span>
            <span class="n">history</span><span class="p">[:,</span> <span class="n">i</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">closeness_len</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">period_len</span> <span class="o">+</span> <span class="n">t</span><span class="p">]</span> <span class="o">=</span> <span class="n">trend</span><span class="p">[:,</span> <span class="n">node</span><span class="p">[</span><span class="n">i</span><span class="p">],</span> <span class="n">t</span><span class="p">,</span> <span class="o">-</span><span class="mi">1</span><span class="p">]</span>
    <span class="n">history</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">expand_dims</span><span class="p">(</span><span class="n">history</span><span class="p">,</span> <span class="mi">3</span><span class="p">)</span>
    <span class="k">return</span> <span class="n">history</span></div>


<div class="viewcode-block" id="TransferDataLoader"><a class="viewcode-back" href="../../../UCTB.dataset.html#UCTB.dataset.data_loader.TransferDataLoader">[docs]</a><span class="k">class</span> <span class="nc">TransferDataLoader</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>

    <span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">sd_params</span><span class="p">,</span> <span class="n">td_params</span><span class="p">,</span> <span class="n">model_params</span><span class="p">,</span> <span class="n">td_data_length</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>

        <span class="k">if</span> <span class="n">td_data_length</span><span class="p">:</span>
            <span class="n">td_params</span><span class="o">.</span><span class="n">update</span><span class="p">({</span><span class="s1">&#39;train_data_length&#39;</span><span class="p">:</span> <span class="n">td_data_length</span><span class="p">})</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">sd_loader</span> <span class="o">=</span> <span class="n">NodeTrafficLoader</span><span class="p">(</span><span class="o">**</span><span class="n">sd_params</span><span class="p">,</span> <span class="o">**</span><span class="n">model_params</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">td_loader</span> <span class="o">=</span> <span class="n">NodeTrafficLoader</span><span class="p">(</span><span class="o">**</span><span class="n">td_params</span><span class="p">,</span> <span class="o">**</span><span class="n">model_params</span><span class="p">)</span>

        <span class="n">td_params</span><span class="o">.</span><span class="n">update</span><span class="p">({</span><span class="s1">&#39;train_data_length&#39;</span><span class="p">:</span> <span class="s1">&#39;180&#39;</span><span class="p">})</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">fake_td_loader</span> <span class="o">=</span> <span class="n">NodeTrafficLoader</span><span class="p">(</span><span class="o">**</span><span class="n">td_params</span><span class="p">,</span> <span class="o">**</span><span class="n">model_params</span><span class="p">)</span>

<div class="viewcode-block" id="TransferDataLoader.traffic_sim"><a class="viewcode-back" href="../../../UCTB.dataset.html#UCTB.dataset.data_loader.TransferDataLoader.traffic_sim">[docs]</a>    <span class="k">def</span> <span class="nf">traffic_sim</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>

        <span class="k">assert</span> <span class="bp">self</span><span class="o">.</span><span class="n">sd_loader</span><span class="o">.</span><span class="n">daily_slots</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">td_loader</span><span class="o">.</span><span class="n">daily_slots</span>

        <span class="n">similar_record</span> <span class="o">=</span> <span class="p">[]</span>

        <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sd_loader</span><span class="o">.</span><span class="n">train_data</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">-</span> <span class="bp">self</span><span class="o">.</span><span class="n">td_loader</span><span class="o">.</span><span class="n">train_data</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span>
                       <span class="nb">int</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">sd_loader</span><span class="o">.</span><span class="n">daily_slots</span><span class="p">)):</span>

            <span class="n">sim</span> <span class="o">=</span> <span class="n">cosine_similarity</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">td_loader</span><span class="o">.</span><span class="n">train_data</span><span class="o">.</span><span class="n">transpose</span><span class="p">(),</span>
                                    <span class="bp">self</span><span class="o">.</span><span class="n">sd_loader</span><span class="o">.</span><span class="n">train_data</span><span class="p">[</span><span class="n">i</span><span class="p">:</span><span class="n">i</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">td_loader</span><span class="o">.</span><span class="n">train_data</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]]</span><span class="o">.</span><span class="n">transpose</span><span class="p">())</span>

            <span class="n">max_sim</span><span class="p">,</span> <span class="n">max_index</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">max</span><span class="p">(</span><span class="n">sim</span><span class="p">,</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">),</span> <span class="n">np</span><span class="o">.</span><span class="n">argmax</span><span class="p">(</span><span class="n">sim</span><span class="p">,</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>

            <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">similar_record</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
                <span class="n">similar_record</span> <span class="o">=</span> <span class="p">[[</span><span class="n">max_sim</span><span class="p">[</span><span class="n">e</span><span class="p">],</span> <span class="n">max_index</span><span class="p">[</span><span class="n">e</span><span class="p">],</span> <span class="n">i</span><span class="p">,</span> <span class="n">i</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">td_loader</span><span class="o">.</span><span class="n">train_data</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]]</span>
                                  <span class="k">for</span> <span class="n">e</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">max_sim</span><span class="p">))]</span>
            <span class="k">else</span><span class="p">:</span>
                <span class="k">for</span> <span class="n">index</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">similar_record</span><span class="p">)):</span>
                    <span class="k">if</span> <span class="n">similar_record</span><span class="p">[</span><span class="n">index</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span> <span class="o">&lt;</span> <span class="n">max_sim</span><span class="p">[</span><span class="n">index</span><span class="p">]:</span>
                        <span class="n">similar_record</span><span class="p">[</span><span class="n">index</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span><span class="n">max_sim</span><span class="p">[</span><span class="n">index</span><span class="p">],</span> <span class="n">max_index</span><span class="p">[</span><span class="n">index</span><span class="p">],</span> <span class="n">i</span><span class="p">,</span>
                                                 <span class="n">i</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">td_loader</span><span class="o">.</span><span class="n">train_data</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]]</span>

        <span class="k">return</span> <span class="n">similar_record</span></div>

<div class="viewcode-block" id="TransferDataLoader.traffic_sim_fake"><a class="viewcode-back" href="../../../UCTB.dataset.html#UCTB.dataset.data_loader.TransferDataLoader.traffic_sim_fake">[docs]</a>    <span class="k">def</span> <span class="nf">traffic_sim_fake</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>

        <span class="k">assert</span> <span class="bp">self</span><span class="o">.</span><span class="n">sd_loader</span><span class="o">.</span><span class="n">daily_slots</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">fake_td_loader</span><span class="o">.</span><span class="n">daily_slots</span>

        <span class="n">similar_record</span> <span class="o">=</span> <span class="p">[]</span>

        <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">sd_loader</span><span class="o">.</span><span class="n">train_data</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">-</span> <span class="bp">self</span><span class="o">.</span><span class="n">fake_td_loader</span><span class="o">.</span><span class="n">train_data</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span>
                       <span class="nb">int</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">sd_loader</span><span class="o">.</span><span class="n">daily_slots</span><span class="p">)):</span>

            <span class="n">sim</span> <span class="o">=</span> <span class="n">cosine_similarity</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">fake_td_loader</span><span class="o">.</span><span class="n">train_data</span><span class="o">.</span><span class="n">transpose</span><span class="p">(),</span>
                                    <span class="bp">self</span><span class="o">.</span><span class="n">sd_loader</span><span class="o">.</span><span class="n">train_data</span><span class="p">[</span>
                                    <span class="n">i</span><span class="p">:</span><span class="n">i</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">fake_td_loader</span><span class="o">.</span><span class="n">train_data</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]]</span><span class="o">.</span><span class="n">transpose</span><span class="p">())</span>

            <span class="n">max_sim</span><span class="p">,</span> <span class="n">max_index</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">max</span><span class="p">(</span><span class="n">sim</span><span class="p">,</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">),</span> <span class="n">np</span><span class="o">.</span><span class="n">argmax</span><span class="p">(</span><span class="n">sim</span><span class="p">,</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>

            <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">similar_record</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
                <span class="n">similar_record</span> <span class="o">=</span> <span class="p">[[</span><span class="n">max_sim</span><span class="p">[</span><span class="n">e</span><span class="p">],</span> <span class="n">max_index</span><span class="p">[</span><span class="n">e</span><span class="p">],</span> <span class="n">i</span><span class="p">,</span> <span class="n">i</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">fake_td_loader</span><span class="o">.</span><span class="n">train_data</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]]</span>
                                  <span class="k">for</span> <span class="n">e</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">max_sim</span><span class="p">))]</span>
            <span class="k">else</span><span class="p">:</span>
                <span class="k">for</span> <span class="n">index</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">similar_record</span><span class="p">)):</span>
                    <span class="k">if</span> <span class="n">similar_record</span><span class="p">[</span><span class="n">index</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span> <span class="o">&lt;</span> <span class="n">max_sim</span><span class="p">[</span><span class="n">index</span><span class="p">]:</span>
                        <span class="n">similar_record</span><span class="p">[</span><span class="n">index</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span><span class="n">max_sim</span><span class="p">[</span><span class="n">index</span><span class="p">],</span> <span class="n">max_index</span><span class="p">[</span><span class="n">index</span><span class="p">],</span> <span class="n">i</span><span class="p">,</span>
                                                 <span class="n">i</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">td_loader</span><span class="o">.</span><span class="n">train_data</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]]</span>

        <span class="k">return</span> <span class="n">similar_record</span></div>

<div class="viewcode-block" id="TransferDataLoader.checkin_sim"><a class="viewcode-back" href="../../../UCTB.dataset.html#UCTB.dataset.data_loader.TransferDataLoader.checkin_sim">[docs]</a>    <span class="k">def</span> <span class="nf">checkin_sim</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>

        <span class="kn">from</span> <span class="nn">sklearn.metrics.pairwise</span> <span class="k">import</span> <span class="n">cosine_similarity</span>

        <span class="n">td_checkin</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="n">e</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="k">for</span> <span class="n">e</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">td_loader</span><span class="o">.</span><span class="n">dataset</span><span class="o">.</span><span class="n">data</span><span class="p">[</span><span class="s1">&#39;ExternalFeature&#39;</span><span class="p">][</span><span class="s1">&#39;CheckInFeature&#39;</span><span class="p">]]</span>
                              <span class="p">)[</span><span class="bp">self</span><span class="o">.</span><span class="n">td_loader</span><span class="o">.</span><span class="n">traffic_data_index</span><span class="p">]</span>
        <span class="n">sd_checkin</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="n">e</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="k">for</span> <span class="n">e</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">sd_loader</span><span class="o">.</span><span class="n">dataset</span><span class="o">.</span><span class="n">data</span><span class="p">[</span><span class="s1">&#39;ExternalFeature&#39;</span><span class="p">][</span><span class="s1">&#39;CheckInFeature&#39;</span><span class="p">]]</span>
                              <span class="p">)[</span><span class="bp">self</span><span class="o">.</span><span class="n">sd_loader</span><span class="o">.</span><span class="n">traffic_data_index</span><span class="p">]</span>

        <span class="n">td_checkin</span> <span class="o">=</span> <span class="n">td_checkin</span> <span class="o">/</span> <span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">max</span><span class="p">(</span><span class="n">td_checkin</span><span class="p">,</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">keepdims</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span> <span class="o">+</span> <span class="mf">0.0001</span><span class="p">)</span>
        <span class="n">sd_checkin</span> <span class="o">=</span> <span class="n">sd_checkin</span> <span class="o">/</span> <span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">max</span><span class="p">(</span><span class="n">sd_checkin</span><span class="p">,</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">keepdims</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span> <span class="o">+</span> <span class="mf">0.0001</span><span class="p">)</span>

        <span class="c1"># cs = cosine_similarity(td_checkin, sd_checkin)</span>

        <span class="c1"># similar_record = [[e[np.argmax(e)], np.argmax(e), ] for e in cs]</span>

        <span class="n">similar_record</span> <span class="o">=</span> <span class="p">[]</span>
        <span class="k">for</span> <span class="n">td_index</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">td_checkin</span><span class="p">)):</span>
            <span class="n">tmp_sim_record</span> <span class="o">=</span> <span class="p">[]</span>
            <span class="k">for</span> <span class="n">sd_index</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">sd_checkin</span><span class="p">)):</span>
                <span class="n">r</span><span class="p">,</span> <span class="n">p</span> <span class="o">=</span> <span class="n">pearsonr</span><span class="p">(</span><span class="n">td_checkin</span><span class="p">[</span><span class="n">td_index</span><span class="p">],</span> <span class="n">sd_checkin</span><span class="p">[</span><span class="n">sd_index</span><span class="p">])</span>
                <span class="n">tmp_sim_record</span><span class="o">.</span><span class="n">append</span><span class="p">([</span><span class="n">r</span><span class="p">,</span> <span class="n">sd_index</span><span class="p">,</span>
                                       <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">sd_loader</span><span class="o">.</span><span class="n">train_y</span><span class="p">)</span> <span class="o">-</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">td_loader</span><span class="o">.</span><span class="n">train_y</span><span class="p">),</span>
                                       <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">sd_loader</span><span class="o">.</span><span class="n">train_y</span><span class="p">)])</span>
            <span class="n">similar_record</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="nb">max</span><span class="p">(</span><span class="n">tmp_sim_record</span><span class="p">,</span> <span class="n">key</span><span class="o">=</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">x</span><span class="p">[</span><span class="mi">0</span><span class="p">]))</span>

        <span class="k">return</span> <span class="n">similar_record</span></div>

<div class="viewcode-block" id="TransferDataLoader.checkin_sim_sd"><a class="viewcode-back" href="../../../UCTB.dataset.html#UCTB.dataset.data_loader.TransferDataLoader.checkin_sim_sd">[docs]</a>    <span class="k">def</span> <span class="nf">checkin_sim_sd</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>

        <span class="n">sd_checkin</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="n">e</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="k">for</span> <span class="n">e</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">sd_loader</span><span class="o">.</span><span class="n">dataset</span><span class="o">.</span><span class="n">data</span><span class="p">[</span><span class="s1">&#39;ExternalFeature&#39;</span><span class="p">][</span><span class="s1">&#39;CheckInFeature&#39;</span><span class="p">]]</span>
                              <span class="p">)[</span><span class="bp">self</span><span class="o">.</span><span class="n">sd_loader</span><span class="o">.</span><span class="n">traffic_data_index</span><span class="p">]</span>
        <span class="n">sd_checkin</span> <span class="o">=</span> <span class="n">sd_checkin</span> <span class="o">/</span> <span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">max</span><span class="p">(</span><span class="n">sd_checkin</span><span class="p">,</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">keepdims</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span> <span class="o">+</span> <span class="mf">0.0001</span><span class="p">)</span>

        <span class="n">cs</span> <span class="o">=</span> <span class="n">cosine_similarity</span><span class="p">(</span><span class="n">sd_checkin</span><span class="p">,</span> <span class="n">sd_checkin</span><span class="p">)</span> <span class="o">-</span> <span class="n">np</span><span class="o">.</span><span class="n">eye</span><span class="p">(</span><span class="n">sd_checkin</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span>

        <span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="n">np</span><span class="o">.</span><span class="n">argmax</span><span class="p">(</span><span class="n">e</span><span class="p">)</span> <span class="k">for</span> <span class="n">e</span> <span class="ow">in</span> <span class="n">cs</span><span class="p">],</span> <span class="n">np</span><span class="o">.</span><span class="n">int32</span><span class="p">)</span></div>

<div class="viewcode-block" id="TransferDataLoader.poi_sim"><a class="viewcode-back" href="../../../UCTB.dataset.html#UCTB.dataset.data_loader.TransferDataLoader.poi_sim">[docs]</a>    <span class="k">def</span> <span class="nf">poi_sim</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>

        <span class="kn">from</span> <span class="nn">sklearn.metrics.pairwise</span> <span class="k">import</span> <span class="n">cosine_similarity</span>

        <span class="n">td_checkin</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="n">e</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> <span class="k">for</span> <span class="n">e</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">td_loader</span><span class="o">.</span><span class="n">dataset</span><span class="o">.</span><span class="n">data</span><span class="p">[</span><span class="s1">&#39;ExternalFeature&#39;</span><span class="p">][</span><span class="s1">&#39;CheckInFeature&#39;</span><span class="p">]]</span>
                              <span class="p">)[</span><span class="bp">self</span><span class="o">.</span><span class="n">td_loader</span><span class="o">.</span><span class="n">traffic_data_index</span><span class="p">]</span>
        <span class="n">sd_checkin</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="n">e</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> <span class="k">for</span> <span class="n">e</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">sd_loader</span><span class="o">.</span><span class="n">dataset</span><span class="o">.</span><span class="n">data</span><span class="p">[</span><span class="s1">&#39;ExternalFeature&#39;</span><span class="p">][</span><span class="s1">&#39;CheckInFeature&#39;</span><span class="p">]]</span>
                              <span class="p">)[</span><span class="bp">self</span><span class="o">.</span><span class="n">sd_loader</span><span class="o">.</span><span class="n">traffic_data_index</span><span class="p">]</span>

        <span class="k">return</span> <span class="p">[[</span><span class="n">e</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">argmax</span><span class="p">(</span><span class="n">e</span><span class="p">)],</span> <span class="n">np</span><span class="o">.</span><span class="n">argmax</span><span class="p">(</span><span class="n">e</span><span class="p">),</span> <span class="p">]</span> <span class="k">for</span> <span class="n">e</span> <span class="ow">in</span> <span class="n">cosine_similarity</span><span class="p">(</span><span class="n">td_checkin</span><span class="p">,</span> <span class="n">sd_checkin</span><span class="p">)]</span></div></div>
</pre></div>

          </div>
        </div>
      </div>
      <div class="sphinxsidebar" role="navigation" aria-label="main navigation">
        <div class="sphinxsidebarwrapper">
<div id="searchbox" style="display: none" role="search">
  <h3 id="searchlabel">Quick search</h3>
    <div class="searchformwrapper">
    <form class="search" action="../../../search.html" method="get">
      <input type="text" name="q" aria-labelledby="searchlabel" />
      <input type="submit" value="Go" />
    </form>
    </div>
</div>
<script type="text/javascript">$('#searchbox').show(0);</script>
        </div>
      </div>
      <div class="clearer"></div>
    </div>
    <div class="related" role="navigation" aria-label="related navigation">
      <h3>Navigation</h3>
      <ul>
        <li class="right" style="margin-right: 10px">
          <a href="../../../genindex.html" title="General Index"
             >index</a></li>
        <li class="right" >
          <a href="../../../py-modindex.html" title="Python Module Index"
             >modules</a> |</li>
        <li class="nav-item nav-item-0"><a href="../../../index.html">UCTB  documentation</a> &#187;</li>
          <li class="nav-item nav-item-1"><a href="../../index.html" >Module code</a> &#187;</li> 
      </ul>
    </div>
    <div class="footer" role="contentinfo">
        &#169; Copyright 2019, Di Chai, Leye Wang, Jin Xu, Wenjie Yang, Liyue Chen.
      Created using <a href="http://sphinx-doc.org/">Sphinx</a> 2.2.1.
    </div>
  </body>
</html>